{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# [HW2] Problem: Blair and their giant peaches\n",
                "\n",
                "Import necessary Python packages.\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import matplotlib.pyplot as plt\n",
                "import numpy as np\n",
                "import scipy.io as spio\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Part (b)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Hint: we provide this function for solving the linear regression problem\n",
                "# There is numpy.linalg.lstsq, whicn you should use outside of this classs\n",
                "\n",
                "def lstsq(A, b):\n",
                "    return np.linalg.solve(A.T @ A, A.T @ b)\n",
                "\n",
                "\n",
                "def main():\n",
                "    # Loading data\n",
                "    data = spio.loadmat('1D_poly.mat', squeeze_me=True)\n",
                "    x_train = np.array(data['x_train'])\n",
                "    y_train = np.array(data['y_train']).T\n",
                "\n",
                "    n = 20  # max degree\n",
                "    err = np.zeros(n-1 )\n",
                "\n",
                "    # TODO: Your code here to fill in err\n",
                "    ### start 1 ###\n",
                "\n",
                "    ### end 1 ###\n",
                "    plt.plot(range(1,n),err)\n",
                "    plt.xlabel('Degree of Polynomial')\n",
                "    plt.ylabel('Training Error')\n",
                "    plt.show()\n",
                "\n",
                "\n",
                "if __name__ == \"__main__\":\n",
                "    main()\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Part (d)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "def main():\n",
                "    data = spio.loadmat('1D_poly.mat', squeeze_me=True)\n",
                "    x_train = np.array(data['x_train'])\n",
                "    y_train = np.array(data['y_train']).T\n",
                "    y_fresh = np.array(data['y_fresh']).T\n",
                "\n",
                "    n = 20  # max degree\n",
                "    err_train = np.zeros(n - 1)\n",
                "    err_fresh = np.zeros(n - 1)\n",
                "\n",
                "    # Recall what we learned from the lecture - why do we need to validate on the ``fresh data''?\n",
                "    # TODO: Your code here to fill in err_fresh and err_train\n",
                "    ### start 2 ###\n",
                "\n",
                "    ### end 2 ###\n",
                "\n",
                "    plt.figure()\n",
                "    plt.ylim([0, 6])\n",
                "    plt.plot(err_train, label='train')\n",
                "    plt.plot(err_fresh, label='fresh')\n",
                "    plt.legend()\n",
                "    plt.show()\n",
                "\n",
                "\n",
                "if __name__ == \"__main__\":\n",
                "    main()\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Part (g)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "data = spio.loadmat('polynomial_regression_samples.mat', squeeze_me=True)\n",
                "data_x = data['x']\n",
                "data_y = data['y']\n",
                "Kc = 4  # 4-fold cross validation\n",
                "KD = 5  # max D = 5\n",
                "LAMBDA = [0, 0.05, 0.1, 0.15, 0.2]\n",
                "\n",
                "feat_x = 0\n",
                "\n",
                "\n",
                "def lstsq(A, b, lambda_=0):\n",
                "    return np.linalg.solve(A.T @ A + lambda_ * np.eye(A.shape[1]), A.T @ b)\n",
                "\n",
                "\n",
                "def assemble_feature(x, D):\n",
                "    n_feature = x.shape[1]\n",
                "    Q = [(np.ones(x.shape[0]), 0, 0)]\n",
                "    i = 0\n",
                "    while Q[i][1] < D:\n",
                "        cx, degree, last_index = Q[i]\n",
                "        for j in range(last_index, n_feature):\n",
                "            Q.append((cx * x[:, j], degree + 1, j))\n",
                "        i += 1\n",
                "    return np.column_stack([q[0] for q in Q])\n",
                "\n",
                "\n",
                "def fit(D, lambda_):\n",
                "    # TODO: YOUR CODE TO COMPUTE THE AVERAGE ERROR PER SAMPLE\n",
                "    ### start 3 ###\n",
                "\n",
                "    ### end 3 ###\n",
                "    pass\n",
                "\n",
                "\n",
                "def main():\n",
                "    np.set_printoptions(precision=11)\n",
                "    Etrain = np.zeros((KD, len(LAMBDA)))\n",
                "    Evalid = np.zeros((KD, len(LAMBDA)))\n",
                "    for D in range(KD):\n",
                "        global feat_x\n",
                "        feat_x = assemble_feature(data_x, D + 1)\n",
                "        print(feat_x.shape)\n",
                "        for i in range(len(LAMBDA)):\n",
                "            Etrain[D, i], Evalid[D, i] = fit(D + 1, LAMBDA[i])\n",
                "\n",
                "    print('Average train error:', Etrain, sep='\\n')\n",
                "    print('Average valid error:', Evalid, sep='\\n')\n",
                "\n",
                "    # TODO: YOUR CODE to find best D and lambda\n",
                "    ### start 4 ###\n",
                "\n",
                "    ### end 4 ###\n",
                "\n",
                "\n",
                "if __name__ == \"__main__\":\n",
                "    main()\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.6.3"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 4
}